

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>kospeech.models.seq2seq.decoder &mdash; KoSpeech 0.0 documentation</title>
  

  
  
  
  

  
  <script type="text/javascript" src="../../../../_static/js/modernizr.min.js"></script>
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
    
    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

    

  
  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../../index.html" class="icon icon-home"> KoSpeech
          

          
          </a>

          
            
            
              <div class="version">
                0.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">NOTES</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/intro.html">Intro</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/Preparation.html">Preparation before Training</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/opts.html">Options</a></li>
</ul>
<p class="caption"><span class="caption-text">ARCHITECTURE</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../Seq2seq.html">Seq2seq</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Transformer.html">Transformer</a></li>
</ul>
<p class="caption"><span class="caption-text">PACKAGE REFERENCE</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../Checkpoint.html">Checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Data.html">Data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Decode.html">Decode</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Evaluator.html">Evaluator</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Optim.html">Optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Trainer.html">Trainer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../Etc.html">Etc</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">KoSpeech</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>kospeech.models.seq2seq.decoder</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for kospeech.models.seq2seq.decoder</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">random</span>
<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">torch</span> <span class="k">import</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">LongTensor</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="k">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Tuple</span>
<span class="kn">from</span> <span class="nn">kospeech.models.seq2seq.attention</span> <span class="k">import</span> <span class="n">LocationAwareAttention</span><span class="p">,</span> <span class="n">MultiHeadAttention</span><span class="p">,</span> <span class="n">AdditiveAttention</span><span class="p">,</span> \
    <span class="n">ScaledDotProductAttention</span>
<span class="kn">from</span> <span class="nn">kospeech.models.modules</span> <span class="k">import</span> <span class="n">Linear</span>
<span class="kn">from</span> <span class="nn">kospeech.models.seq2seq.sublayers</span> <span class="k">import</span> <span class="n">BaseRNN</span>
<span class="kn">from</span> <span class="nn">kospeech.models.transformer.sublayers</span> <span class="k">import</span> <span class="n">AddNorm</span>


<span class="k">def</span> <span class="nf">_inflate</span><span class="p">(</span><span class="n">tensor</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">n_repeat</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot; Given a tensor, &#39;inflates&#39; it along the given dimension by replicating each slice specified number of times  &quot;&quot;&quot;</span>
    <span class="n">repeat_dims</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">tensor</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
    <span class="n">repeat_dims</span><span class="p">[</span><span class="n">dim</span><span class="p">]</span> <span class="o">*=</span> <span class="n">n_repeat</span>

    <span class="k">return</span> <span class="n">tensor</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="o">*</span><span class="n">repeat_dims</span><span class="p">)</span>


<div class="viewcode-block" id="Seq2seqDecoder"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqDecoder">[docs]</a><span class="k">class</span> <span class="nc">Seq2seqDecoder</span><span class="p">(</span><span class="n">BaseRNN</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Converts higher level features (from encoder) into output utterances</span>
<span class="sd">    by specifying a probability distribution over sequences of characters.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_classes (int): number of classfication</span>
<span class="sd">        max_length (int): a maximum allowed length for the sequence to be processed</span>
<span class="sd">        hidden_dim (int): dimension of RNN`s hidden state vector</span>
<span class="sd">        sos_id (int): index of the start of sentence symbol</span>
<span class="sd">        eos_id (int): index of the end of sentence symbol</span>
<span class="sd">        attn_mechanism (str): type of attention mechanism (default: dot)</span>
<span class="sd">        num_heads (int): number of attention heads. (default: 4)</span>
<span class="sd">        num_layers (int, optional): number of recurrent layers (default: 1)</span>
<span class="sd">        rnn_type (str, optional): type of RNN cell (default: lstm)</span>
<span class="sd">        dropout_p (float, optional): dropout probability (default: 0.3)</span>
<span class="sd">        device (torch.device): device - &#39;cuda&#39; or &#39;cpu&#39;</span>

<span class="sd">    Inputs: inputs, encoder_outputs, teacher_forcing_ratio, return_decode_dict</span>
<span class="sd">        - **inputs** (batch, seq_len, input_size): list of sequences, whose length is the batch size and within which</span>
<span class="sd">          each sequence is a list of token IDs.  It is used for teacher forcing when provided. (default `None`)</span>
<span class="sd">        - **encoder_outputs** (batch, seq_len, hidden_dim): tensor with containing the outputs of the listener.</span>
<span class="sd">          Used for attention mechanism (default is `None`).</span>
<span class="sd">        - **teacher_forcing_ratio** (float): The probability that teacher forcing will be used. A random number is</span>
<span class="sd">          drawn uniformly from 0-1 for every decoding token, and if the sample is smaller than the given value,</span>
<span class="sd">          teacher forcing would be used (default is 0).</span>
<span class="sd">        - **return_decode_dict** (dict): dictionary which contains decode informations.</span>

<span class="sd">    Returns: decoder_outputs, decode_dict</span>
<span class="sd">        - **decoder_outputs** (seq_len, batch, num_classes): list of tensors containing</span>
<span class="sd">          the outputs of the decoding function.</span>
<span class="sd">        - **decode_dict**: dictionary containing additional information as follows {*KEY_ATTENTION_SCORE* : list of scores</span>
<span class="sd">          representing encoder outputs, *KEY_SEQUENCE_SYMBOL* : list of sequences, where each sequence is a list of</span>
<span class="sd">          predicted token IDs }.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">KEY_ATTENTION_SCORE</span> <span class="o">=</span> <span class="s1">&#39;attention_score&#39;</span>
    <span class="n">KEY_LENGTH</span> <span class="o">=</span> <span class="s1">&#39;length&#39;</span>
    <span class="n">KEY_SEQUENCE_SYMBOL</span> <span class="o">=</span> <span class="s1">&#39;sequence_symbol&#39;</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_classes</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>                    <span class="c1"># number of classfication</span>
                 <span class="n">max_length</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">120</span><span class="p">,</span>               <span class="c1"># a maximum allowed length for the sequence to be processed</span>
                 <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span><span class="p">,</span>              <span class="c1"># dimension of RNN`s hidden state vector</span>
                 <span class="n">sos_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>                     <span class="c1"># start of sentence token`s id</span>
                 <span class="n">eos_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>                     <span class="c1"># end of sentence token`s id</span>
                 <span class="n">attn_mechanism</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;multi-head&#39;</span><span class="p">,</span>  <span class="c1"># type of attention mechanism</span>
                 <span class="n">num_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>                  <span class="c1"># number of attention heads</span>
                 <span class="n">num_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>                 <span class="c1"># number of RNN layers</span>
                 <span class="n">rnn_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;lstm&#39;</span><span class="p">,</span>              <span class="c1"># type of RNN cell</span>
                 <span class="n">dropout_p</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.3</span><span class="p">,</span>              <span class="c1"># dropout probability</span>
                 <span class="n">device</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;cuda&#39;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>       <span class="c1"># device - &#39;cuda&#39; or &#39;cpu&#39;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Seq2seqDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="n">hidden_dim</span><span class="p">,</span> <span class="n">num_layers</span><span class="p">,</span> <span class="n">rnn_type</span><span class="p">,</span> <span class="n">dropout_p</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_classes</span> <span class="o">=</span> <span class="n">num_classes</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_heads</span> <span class="o">=</span> <span class="n">num_heads</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span> <span class="o">=</span> <span class="n">max_length</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span> <span class="o">=</span> <span class="n">eos_id</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sos_id</span> <span class="o">=</span> <span class="n">sos_id</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">acoutsic_weight</span> <span class="o">=</span> <span class="mf">0.9</span>  <span class="c1"># acoustic model weight</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">language_weight</span> <span class="o">=</span> <span class="mf">0.1</span>  <span class="c1"># language model weight</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">=</span> <span class="n">attn_mechanism</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">num_classes</span><span class="p">,</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_dropout</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">dropout_p</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;loc&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">attention</span> <span class="o">=</span> <span class="n">AddNorm</span><span class="p">(</span><span class="n">LocationAwareAttention</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="n">smoothing</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;multi-head&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">attention</span> <span class="o">=</span> <span class="n">AddNorm</span><span class="p">(</span><span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">),</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;additive&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">attention</span> <span class="o">=</span> <span class="n">AddNorm</span><span class="p">(</span><span class="n">AdditiveAttention</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">),</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;scaled-dot&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">attention</span> <span class="o">=</span> <span class="n">AddNorm</span><span class="p">(</span><span class="n">ScaledDotProductAttention</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">),</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Unsupported attention: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">attn_mechanism</span><span class="p">))</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">projection</span> <span class="o">=</span> <span class="n">AddNorm</span><span class="p">(</span><span class="n">Linear</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="n">hidden_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">hidden_dim</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">Linear</span><span class="p">(</span><span class="n">hidden_dim</span><span class="p">,</span> <span class="n">num_classes</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_var</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">hidden</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span>
                     <span class="n">encoder_outputs</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">attn</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span> <span class="n">Tensor</span><span class="p">]:</span>
        <span class="n">batch_size</span><span class="p">,</span> <span class="n">output_lengths</span> <span class="o">=</span> <span class="n">input_var</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="n">input_var</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="n">embedded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">input_var</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
        <span class="n">embedded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dropout</span><span class="p">(</span><span class="n">embedded</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="o">.</span><span class="n">flatten_parameters</span><span class="p">()</span>

        <span class="n">output</span><span class="p">,</span> <span class="n">hidden</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">(</span><span class="n">embedded</span><span class="p">,</span> <span class="n">hidden</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;loc&#39;</span><span class="p">:</span>
            <span class="n">context</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">attention</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">attn</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">context</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">attention</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">)</span>

        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(</span><span class="n">context</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span><span class="p">))</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span><span class="p">)</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">output</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span><span class="p">))</span>

        <span class="n">step_output</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">log_softmax</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">step_output</span> <span class="o">=</span> <span class="n">step_output</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">output_lengths</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">return</span> <span class="n">step_output</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span>

<div class="viewcode-block" id="Seq2seqDecoder.forward"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqDecoder.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
                <span class="n">teacher_forcing_ratio</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">language_model</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
                <span class="n">return_decode_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="nb">dict</span><span class="p">]:</span>
        <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>
        <span class="n">result</span><span class="p">,</span> <span class="n">decode_dict</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(),</span> <span class="nb">dict</span><span class="p">()</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
            <span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_ATTENTION_SCORE</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
            <span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_SEQUENCE_SYMBOL</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>

        <span class="n">inputs</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">max_length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_args</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">teacher_forcing_ratio</span><span class="p">,</span> <span class="n">language_model</span><span class="p">)</span>
        <span class="n">use_teacher_forcing</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">if</span> <span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">()</span> <span class="o">&lt;</span> <span class="n">teacher_forcing_ratio</span> <span class="k">else</span> <span class="kc">False</span>
        <span class="n">use_language_model</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">if</span> <span class="n">language_model</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="kc">False</span>
        <span class="n">lengths</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">max_length</span><span class="p">]</span> <span class="o">*</span> <span class="n">batch_size</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">use_teacher_forcing</span><span class="p">:</span>
            <span class="n">inputs</span> <span class="o">=</span> <span class="n">inputs</span><span class="p">[</span><span class="n">inputs</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span><span class="p">]</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;loc&#39;</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_mechanism</span> <span class="o">==</span> <span class="s1">&#39;additive&#39;</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">inputs</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)):</span>
                    <span class="n">input_var</span> <span class="o">=</span> <span class="n">inputs</span><span class="p">[:,</span> <span class="n">di</span><span class="p">]</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
                    <span class="n">step_output</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward_step</span><span class="p">(</span><span class="n">input_var</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">attn</span><span class="p">)</span>
                    <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">step_output</span><span class="p">)</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="n">step_outputs</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward_step</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">attn</span><span class="p">)</span>

                <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">step_outputs</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)):</span>
                    <span class="n">step_output</span> <span class="o">=</span> <span class="n">step_outputs</span><span class="p">[:,</span> <span class="n">di</span><span class="p">,</span> <span class="p">:]</span>
                    <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">step_output</span><span class="p">)</span>

        <span class="k">else</span><span class="p">:</span>
            <span class="n">input_var</span> <span class="o">=</span> <span class="n">inputs</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
            <span class="n">prev_tokens</span> <span class="o">=</span> <span class="n">input_var</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span> <span class="k">if</span> <span class="n">use_language_model</span> <span class="k">else</span> <span class="kc">None</span>

            <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">max_length</span><span class="p">):</span>
                <span class="n">step_output</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward_step</span><span class="p">(</span><span class="n">input_var</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="n">attn</span><span class="p">)</span>
                <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">step_output</span><span class="p">)</span>
                <span class="n">input_var</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>

                <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
                    <span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_ATTENTION_SCORE</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">attn</span><span class="p">)</span>
                    <span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_SEQUENCE_SYMBOL</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">input_var</span><span class="p">)</span>
                    <span class="n">eos_batches</span> <span class="o">=</span> <span class="n">input_var</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">eq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span><span class="p">)</span>

                    <span class="k">if</span> <span class="n">eos_batches</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                        <span class="n">eos_batches</span> <span class="o">=</span> <span class="n">eos_batches</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
                        <span class="n">update_idx</span> <span class="o">=</span> <span class="p">((</span><span class="n">lengths</span> <span class="o">&gt;</span> <span class="n">di</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">eos_batches</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span>
                        <span class="n">lengths</span><span class="p">[</span><span class="n">update_idx</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_SEQUENCE_SYMBOL</span><span class="p">])</span>

                    <span class="k">if</span> <span class="n">use_language_model</span><span class="p">:</span>
                        <span class="n">lm_step_output</span> <span class="o">=</span> <span class="n">language_model</span><span class="o">.</span><span class="n">forward_step</span><span class="p">(</span><span class="n">prev_tokens</span><span class="p">,</span> <span class="kc">None</span><span class="p">)[</span><span class="mi">0</span><span class="p">][:,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:]</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
                        <span class="n">step_output</span> <span class="o">=</span> <span class="n">step_output</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">acoustic_weight</span> <span class="o">+</span> <span class="n">lm_step_output</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">language_weight</span>
                        <span class="n">prev_tokens</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">prev_tokens</span><span class="p">,</span> <span class="n">step_output</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]],</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">return_decode_dict</span><span class="p">:</span>
            <span class="n">decode_dict</span><span class="p">[</span><span class="n">Seq2seqDecoder</span><span class="o">.</span><span class="n">KEY_LENGTH</span><span class="p">]</span> <span class="o">=</span> <span class="n">lengths</span>
            <span class="n">result</span> <span class="o">=</span> <span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">decode_dict</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">del</span> <span class="n">decode_dict</span>

        <span class="k">return</span> <span class="n">result</span></div>

<div class="viewcode-block" id="Seq2seqDecoder.validate_args"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqDecoder.validate_args">[docs]</a>    <span class="k">def</span> <span class="nf">validate_args</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">:</span> <span class="n">Tensor</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
                      <span class="n">teacher_forcing_ratio</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
                      <span class="n">language_model</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
        <span class="sd">&quot;&quot;&quot; Validate arguments &quot;&quot;&quot;</span>
        <span class="k">assert</span> <span class="n">encoder_outputs</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
        <span class="n">batch_size</span> <span class="o">=</span> <span class="n">encoder_outputs</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">inputs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>  <span class="c1"># inference</span>
            <span class="n">inputs</span> <span class="o">=</span> <span class="n">LongTensor</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">sos_id</span><span class="p">]</span> <span class="o">*</span> <span class="n">batch_size</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
            <span class="n">max_length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span>

            <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
                <span class="n">inputs</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>

            <span class="k">if</span> <span class="n">teacher_forcing_ratio</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Teacher forcing has to be disabled (set 0) when no inputs is provided.&quot;</span><span class="p">)</span>

        <span class="k">else</span><span class="p">:</span>
            <span class="n">max_length</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>  <span class="c1"># minus the start of sequence symbol</span>

        <span class="k">if</span> <span class="n">language_model</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">language_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">max_length</span></div></div>


<div class="viewcode-block" id="Seq2seqTopKDecoder"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqTopKDecoder">[docs]</a><span class="k">class</span> <span class="nc">Seq2seqTopKDecoder</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Top-K decoding with beam search.</span>

<span class="sd">    Args:</span>
<span class="sd">        decoder (Seq2seqGreedyDecoder): decoder to which beam search will be applied</span>
<span class="sd">        beam_size (int): size of beam</span>

<span class="sd">    Inputs: input_var, encoder_outputs</span>
<span class="sd">        - **input_var** : sequence of sos_id</span>
<span class="sd">        - **encoder_outputs** : tensor containing the encoded features of the input sequence</span>

<span class="sd">    Returns: decoder_outputs</span>
<span class="sd">        - **decoder_outputs** :  list of tensors containing the outputs of the decoding function.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">decoder</span><span class="p">:</span> <span class="n">Seq2seqDecoder</span><span class="p">,</span> <span class="n">beam_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Seq2seqTopKDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_classes</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">num_classes</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">max_length</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">hidden_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">forward_step</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">forward_step</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">validate_args</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">validate_args</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">decoder</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pos_index</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span> <span class="o">=</span> <span class="n">beam_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sos_id</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">sos_id</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">eos_id</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">min_length</span> <span class="o">=</span> <span class="mi">5</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">=</span> <span class="mf">1.2</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">device</span>

<div class="viewcode-block" id="Seq2seqTopKDecoder.forward"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqTopKDecoder.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_var</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">:</span> <span class="n">Tensor</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">:</span>
        <span class="n">inputs</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">max_length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_args</span><span class="p">(</span><span class="n">input_var</span><span class="p">,</span> <span class="n">encoder_outputs</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pos_index</span> <span class="o">=</span> <span class="p">(</span><span class="n">LongTensor</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">batch_size</span><span class="p">))</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>

        <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>
        <span class="n">inflated_encoder_outputs</span> <span class="o">=</span> <span class="n">_inflate</span><span class="p">(</span><span class="n">encoder_outputs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>

        <span class="c1"># Initialize the scores; for the first step,</span>
        <span class="c1"># ignore the inflated copies to avoid duplicate entries in the top k</span>
        <span class="c1"># sequence_scores: tensor([[-inf], [-inf], [-inf], [-inf], [-inf], [-inf], [-inf], [-inf]])</span>
        <span class="n">sequence_scores</span> <span class="o">=</span> <span class="n">encoder_outputs</span><span class="o">.</span><span class="n">new_zeros</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
        <span class="n">sequence_scores</span><span class="o">.</span><span class="n">fill_</span><span class="p">(</span><span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;Inf&#39;</span><span class="p">))</span>

        <span class="c1"># If beam_size is three, tensor([ 0,  3,  6,  9, 12, 15, 18, 21])</span>
        <span class="c1"># sequence_scores: tensor([[0.], [-inf], [-inf], [0.], [-inf], [-inf], [0.], [-inf] ..])  BKx1</span>
        <span class="n">fill_index</span> <span class="o">=</span> <span class="n">LongTensor</span><span class="p">([</span><span class="n">i</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">batch_size</span><span class="p">)])</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
        <span class="n">sequence_scores</span><span class="o">.</span><span class="n">index_fill_</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">fill_index</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">)</span>

        <span class="c1"># Initialize the input vector</span>
        <span class="n">input_var</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">LongTensor</span><span class="p">([[</span><span class="bp">self</span><span class="o">.</span><span class="n">sos_id</span><span class="p">]</span> <span class="o">*</span> <span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">]),</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># 1xBK =&gt; BKx1</span>
        <span class="n">input_var</span> <span class="o">=</span> <span class="n">input_var</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>

        <span class="c1"># Store decisions for backtracking</span>
        <span class="n">stored_outputs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
        <span class="n">stored_scores</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
        <span class="n">stored_predecessors</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
        <span class="n">stored_emitted_symbols</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>

        <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">max_length</span><span class="p">):</span>
            <span class="c1"># Run the RNN one step forward</span>
            <span class="n">step_output</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward_step</span><span class="p">(</span><span class="n">input_var</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">inflated_encoder_outputs</span><span class="p">,</span> <span class="n">attn</span><span class="p">)</span>
            <span class="n">stored_outputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">step_output</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>

            <span class="c1"># force the output to be longer than self.min_length</span>
            <span class="k">if</span> <span class="n">di</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_length</span><span class="p">:</span>
                <span class="n">step_output</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;Inf&#39;</span><span class="p">)</span>

            <span class="n">sequence_scores</span> <span class="o">=</span> <span class="n">_inflate</span><span class="p">(</span><span class="n">sequence_scores</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">num_classes</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># BKx1 =&gt; BKxC</span>
            <span class="n">sequence_scores</span> <span class="o">+=</span> <span class="n">step_output</span>  <span class="c1"># step_output: BKxC, Cumulative Score</span>
            <span class="n">scores</span><span class="p">,</span> <span class="n">candidates</span> <span class="o">=</span> <span class="n">sequence_scores</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># BKxC =&gt; BxKC =&gt; BxK</span>

            <span class="c1"># Reshape input = (bk, 1) and sequence_scores = (bk, 1)</span>
            <span class="n">input_var</span> <span class="o">=</span> <span class="p">(</span><span class="n">candidates</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">num_classes</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># BxK =&gt; BKx1</span>
            <span class="n">sequence_scores</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># BKx1</span>

            <span class="c1"># Update fields for next timestep</span>
            <span class="c1"># self.pos_index.expand_as(candidates)</span>
            <span class="n">predecessors</span> <span class="o">=</span> <span class="p">(</span><span class="n">candidates</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">num_classes</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos_index</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">candidates</span><span class="p">))</span>
            <span class="n">predecessors</span> <span class="o">=</span> <span class="n">predecessors</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># BKx1</span>

            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hidden</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
                <span class="n">hidden</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">([</span><span class="n">h</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">predecessors</span><span class="o">.</span><span class="n">squeeze</span><span class="p">())</span> <span class="k">for</span> <span class="n">h</span> <span class="ow">in</span> <span class="n">hidden</span><span class="p">])</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">hidden</span> <span class="o">=</span> <span class="n">hidden</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">predecessors</span><span class="o">.</span><span class="n">squeeze</span><span class="p">())</span>  <span class="c1"># hidden: LxBKxD</span>
                                                                         <span class="c1"># predecessors: BK</span>

            <span class="c1"># Update sequence scores and erase scores for end-of-sentence symbol so that they aren&#39;t expanded</span>
            <span class="n">stored_scores</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sequence_scores</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_length_penalty</span><span class="p">(</span><span class="n">di</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
            <span class="n">eos_indices</span> <span class="o">=</span> <span class="n">input_var</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">eq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">eos_indices</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">sequence_scores</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">masked_fill_</span><span class="p">(</span><span class="n">eos_indices</span><span class="p">,</span> <span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;Inf&#39;</span><span class="p">))</span>

            <span class="c1"># Cache results for backtracking</span>
            <span class="n">stored_predecessors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">predecessors</span><span class="p">)</span>
            <span class="n">stored_emitted_symbols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">input_var</span><span class="p">)</span>

        <span class="c1"># Do backtracking to return the optimal values</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backtrack</span><span class="p">(</span><span class="n">stored_outputs</span><span class="p">,</span> <span class="n">stored_predecessors</span><span class="p">,</span> <span class="n">stored_emitted_symbols</span><span class="p">,</span> <span class="n">stored_scores</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>

        <span class="n">decoder_outputs</span> <span class="o">=</span> <span class="p">[</span><span class="n">step</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span> <span class="k">for</span> <span class="n">step</span> <span class="ow">in</span> <span class="n">output</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">decoder_outputs</span></div>

    <span class="k">def</span> <span class="nf">_backtrack</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stored_outputs</span><span class="p">,</span> <span class="n">stored_predecessors</span><span class="p">,</span> <span class="n">stored_emitted_symbols</span><span class="p">,</span> <span class="n">stored_scores</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Backtracks over batch to generate optimal k-sequences.</span>
<span class="sd">        Args:</span>
<span class="sd">            stored_outputs [(batch*k, vocab_size)] * sequence_length: A Tensor of outputs from network</span>
<span class="sd">            stored_predecessors [(batch*k)] * sequence_length: A Tensor of predecessors</span>
<span class="sd">            stored_emitted_symbols [(batch*k)] * sequence_length: A Tensor of predicted tokens</span>
<span class="sd">            scores [(batch*k)] * sequence_length: A Tensor containing sequence scores for every token t = [0, ... , seq_len - 1]</span>
<span class="sd">            batch_size: Size of the batch</span>
<span class="sd">        Returns:</span>
<span class="sd">            output [(batch, k, vocab_size)] * sequence_length: A list of the output probabilities (p_n)</span>
<span class="sd">            from the last layer of the RNN, for every n = [0, ... , seq_len - 1]</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="c1"># initialize return variables given different types</span>
        <span class="n">output</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>

        <span class="c1"># the last step output of the beams are not sorted</span>
        <span class="c1"># thus they are sorted here</span>
        <span class="n">sorted_score</span><span class="p">,</span> <span class="n">sorted_idx</span> <span class="o">=</span> <span class="n">stored_scores</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span>  <span class="c1"># BxK</span>
        <span class="c1"># initialize the sequence scores with the sorted last step beam scores</span>
        <span class="n">s</span> <span class="o">=</span> <span class="n">sorted_score</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>

        <span class="c1"># the number of EOS found</span>
        <span class="c1"># in the backward loop below for each batch</span>
        <span class="n">batch_eos_found</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="n">batch_size</span>

        <span class="n">t</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span> <span class="o">-</span> <span class="mi">1</span>
        <span class="c1"># initialize the back pointer with the sorted order of the last step beams.</span>
        <span class="c1"># add self.pos_index for indexing variable with b*k as the first dimension.</span>
        <span class="n">t_predecessors</span> <span class="o">=</span> <span class="p">(</span><span class="n">sorted_idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos_index</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">sorted_idx</span><span class="p">))</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span>  <span class="c1"># BxK =&gt; BK</span>

        <span class="k">while</span> <span class="n">t</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="c1"># Re-order the variables with the back pointer</span>
            <span class="n">current_output</span> <span class="o">=</span> <span class="n">stored_outputs</span><span class="p">[</span><span class="n">t</span><span class="p">]</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">t_predecessors</span><span class="p">)</span>

            <span class="c1"># Re-order the back pointer of the previous step with the back pointer of</span>
            <span class="c1"># the current step</span>
            <span class="n">t_predecessors</span> <span class="o">=</span> <span class="n">stored_predecessors</span><span class="p">[</span><span class="n">t</span><span class="p">]</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">t_predecessors</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>

            <span class="c1"># This tricky block handles dropped sequences that see EOS earlier.</span>
            <span class="c1"># The basic idea is summarized below:</span>
            <span class="c1">#</span>
            <span class="c1">#   Terms:</span>
            <span class="c1">#       Ended sequences = sequences that see EOS early and dropped</span>
            <span class="c1">#       Survived sequences = sequences in the last step of the beams</span>
            <span class="c1">#</span>
            <span class="c1">#       Although the ended sequences are dropped during decoding,</span>
            <span class="c1">#   their generated symbols and complete backtracking information are still</span>
            <span class="c1">#   in the backtracking variables.</span>
            <span class="c1">#   For each batch, everytime we see an EOS in the backtracking process,</span>
            <span class="c1">#       1. If there is survived sequences in the return variables, replace</span>
            <span class="c1">#       the one with the lowest survived sequence score with the new ended</span>
            <span class="c1">#       sequences</span>
            <span class="c1">#       2. Otherwise, replace the ended sequence with the lowest sequence</span>
            <span class="c1">#       score with the new ended sequence</span>
            <span class="c1">#</span>
            <span class="n">eos_indices</span> <span class="o">=</span> <span class="n">stored_emitted_symbols</span><span class="p">[</span><span class="n">t</span><span class="p">]</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">eq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">eos_id</span><span class="p">)</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">eos_indices</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">eos_indices</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">):</span>
                    <span class="c1"># Indices of the EOS symbol for both variables</span>
                    <span class="c1"># with b*k as the first dimension, and b, k for</span>
                    <span class="c1"># the first two dimensions</span>
                    <span class="n">idx</span> <span class="o">=</span> <span class="n">eos_indices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
                    <span class="n">b_idx</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">idx</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span>
                    <span class="c1"># The indices of the replacing position</span>
                    <span class="c1"># according to the replacement strategy noted above</span>
                    <span class="n">res_k_idx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span> <span class="o">-</span> <span class="p">(</span><span class="n">batch_eos_found</span><span class="p">[</span><span class="n">b_idx</span><span class="p">]</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
                    <span class="n">batch_eos_found</span><span class="p">[</span><span class="n">b_idx</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
                    <span class="n">res_idx</span> <span class="o">=</span> <span class="n">b_idx</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span> <span class="o">+</span> <span class="n">res_k_idx</span>

                    <span class="c1"># Replace the old information in return variables</span>
                    <span class="c1"># with the new ended sequence information</span>
                    <span class="n">t_predecessors</span><span class="p">[</span><span class="n">res_idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">stored_predecessors</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="n">idx</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>
                    <span class="n">current_output</span><span class="p">[</span><span class="n">res_idx</span><span class="p">,</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">stored_outputs</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="n">idx</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="p">:]</span>

            <span class="c1"># record the back tracked results</span>
            <span class="n">output</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">current_output</span><span class="p">)</span>

            <span class="n">t</span> <span class="o">-=</span> <span class="mi">1</span>

        <span class="c1"># Sort and re-order again as the added ended sequences may change</span>
        <span class="c1"># the order (very unlikely)</span>
        <span class="n">s</span><span class="p">,</span> <span class="n">re_sorted_idx</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span>
        <span class="n">re_sorted_idx</span> <span class="o">=</span> <span class="p">(</span><span class="n">re_sorted_idx</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos_index</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">re_sorted_idx</span><span class="p">))</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">)</span>

        <span class="c1"># Reverse the sequences and re-order at the same time</span>
        <span class="c1"># It is reversed because the backtracking happens in reverse time order</span>
        <span class="n">output</span> <span class="o">=</span> <span class="p">[</span><span class="n">step</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">re_sorted_idx</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="k">for</span> <span class="n">step</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">output</span><span class="p">)]</span>
        <span class="k">return</span> <span class="n">output</span>

<div class="viewcode-block" id="Seq2seqTopKDecoder.get_length_penalty"><a class="viewcode-back" href="../../../../Seq2seq.html#kospeech.models.seq2seq.decoder.Seq2seqTopKDecoder.get_length_penalty">[docs]</a>    <span class="k">def</span> <span class="nf">get_length_penalty</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Calculate length-penalty.</span>
<span class="sd">        because shorter sentence usually have bigger probability.</span>
<span class="sd">        using alpha = 1.2, min_length = 5 usually.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">min_length</span> <span class="o">+</span> <span class="n">length</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">min_length</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span></div></div>
</pre></div>

           </div>
           
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Soohwan Kim

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>